package com.yufei.infoExtractor.parallel.thread;

import java.io.IOException;

import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.CountDownLatch;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import com.yufei.infoExtractor.cache.InfoExtractorCache;
import com.yufei.infoExtractor.extractor.fetchdecision.DecideByIsHadFetched;
import com.yufei.infoExtractor.extractor.fetchdecision.DecideByUrlSimilarity;
import com.yufei.infoExtractor.pfw.entity.PropertyMatch;
import com.yufei.infoExtractor.pfw.entity.Seedsite;
import com.yufei.infoExtractor.util.CommonUtil;
import com.yufei.infoExtractor.util.EncryptUtil;
import com.yufei.infoExtractor.util.PatternUtils;

/**
 * @author jasstion
   2012-11-23
   url extractor
 *
 */
public class UrlParserThread implements Runnable {
	public static final Log mLog = LogFactory.getLog(UrlParserThread.class);
    private Map context=null;
	private List<String> readyUrls = null;
	private CountDownLatch countDownLatch = null;

	public UrlParserThread(ThreadLocal<Map> threadLocalcontext, List<String> readyUrls,
			CountDownLatch countDownLatch) {
		super();
    this.context=threadLocalcontext.get();
    this.readyUrls = readyUrls;
		this.countDownLatch = countDownLatch;
	}

	

	public List<String> getReadyUrls() {
		return readyUrls;
	}

	public void setReadyUrls(List<String> readyUrls) {
		this.readyUrls = readyUrls;
	}

	public CountDownLatch getCountDownLatch() {
		return countDownLatch;
	}

	public void setCountDownLatch(CountDownLatch countDownLatch) {
		this.countDownLatch = countDownLatch;
	}

	@Override
	public void run() {
		extractorUrls(context);

		countDownLatch.countDown();
		mLog.info("线程执行完毕");
	}
	public void extractorUrls(Map context){
		// TODO Auto-generated method stub
		InfoExtractorCache< String, String> urlCache=(InfoExtractorCache<String, String>) context.get("urlCache");
		Integer currentDepth = (Integer) context.get("currentDepth");
		Integer depth = (Integer) context.get("depth");
		Set[] linkSet = (Set[]) context.get("linkSet");
		Map linkContentCache = (Map) context.get("linkContentCache");
		Set<String> linkFingerPrints = (Set<String>) context
				.get("linkFingerPrints");
		Iterator<String> iter = readyUrls.iterator();
		String htmlContent = null;
		List<PropertyMatch> propertyMatches=(List<PropertyMatch>) context.get("propertyMatches");
		Seedsite seed=(Seedsite) context.get("seed");
		String urlMd5=null;
		String  url1 = null;


		for(String url:readyUrls){
			
			urlMd5= EncryptUtil.md5(url);

			// 将此链接放入链接指纹用于防止此链接被重复处理
			linkFingerPrints.add(urlMd5);
	
			try {
				htmlContent = CommonUtil.getHtmlContent(context, url);
			//	Cache cache=AppUtil.getCacheByCacheName(AppUtil.defaultUrlCache);///seed.getTaskName());
			
			
				urlCache.put(urlMd5,htmlContent);
			} catch (IOException e) {
				// TODO Auto-generated catch block
				PatternUtils.mLog.info("在对链接进行子链接提取的过程中访问+'"+url+"'出错，直接忽略该Url的子链接提取");
				continue;
			}
			if (CommonUtil.isEmptyOrNull(htmlContent)) {
				PatternUtils.mLog.info("链接+'"+url+"'请求返回的内容为空，直接忽略该Url的子链接提取");

				continue;
			}
	
			String filterregexes = CommonUtil.getPropertyMatchFromListByPropertyName("linkFilterRegex", propertyMatches).getMatch();
				
			List<String> filterUrls = PatternUtils.getListStrByRegex(
					htmlContent, filterregexes);
			if (CommonUtil.isEmptyOrNull(filterUrls)) {
				mLog.debug("在对链接：'"+url+"'进行子链接提取的过程中，根据链接匹配正则表达式'"+filterregexes+"'对网页内容：'"+htmlContent+"'匹配失败，请检查网页内容或者是链接匹配表达式是否有误");
				continue;
			}
			int totalValidateUrlCount=0;
			for (String extractedUrl : filterUrls) {
				mLog.debug("开始对子链接:'"+extractedUrl+"'进行链接合法性检测以及链接补全");

				if (!CommonUtil.isValidUrl(extractedUrl)) {
	
					continue;
	
				}
				url1 = CommonUtil.formatUrl(extractedUrl);
				url1 = CommonUtil.normalizeUrl(extractedUrl, url);
				mLog.debug("补全后的链接为'"+url1+"'");
		      
				boolean isOk=new DecideByIsHadFetched().makeDecision(context, url1);

				 isOk=new DecideByUrlSimilarity().makeDecision(context, url1);
				if (!isOk) {
					
					continue;
				}
				totalValidateUrlCount++;
				linkSet[currentDepth + 1].add(url1);
				
	
			}
			mLog.debug("子链接'"+url+"'总共抽取到有效链接个数为：'"+totalValidateUrlCount+"'");


		}
	}

}
